from transformers import Qwen2Tokenizer, Qwen2TokenizerFast

from constant import qwen05_pretrained_model

try:
    tokenizer = Qwen2TokenizerFast.from_pretrained(qwen05_pretrained_model)
    print("using tokenizer fast")
except Exception as e:
    tokenizer = Qwen2Tokenizer.from_pretrained(qwen05_pretrained_model)


text = "你好，你是谁"

token_ids1 = tokenizer.encode(text)
print(token_ids1)
# [108386, 3837, 105043, 100165]
print(len(tokenizer))